"""
Case Type   : 极致RTO-数据库系统类
Case Name   : 开启极致RTO，备机进行switchover过程中，主机异常挂住后恢复，切换成功后，还原主备状态，关闭极致RTO，主备数据一致
Create At   : 2020/10/15
Owner       : @peilinqian
Description :
    1、开启极致RTO
    2、备机进行switchover
    3、主机数据库进程挂住kill - 19；
    4、验证switchover结果
    5、等待30S，恢复主机数据库进程kill - 18；
    6、重启主备；
    7、备机再次进行switchover
    8、还原主备，原主机进行switchover
    9、执行完成后，关闭极致RTO
    10、查询主备数据是否一致
Expect      :
    1、开启极致RTO成功
    2、备机开始进行switchover
    3、主机数据库进程挂住kill - 19成功；
    4、验证switchover结果失败
    5、等待30S，恢复主机数据库进程kill - 18成功；
    6、重启主备成功；
    7、备机再次进行switchover成功
    8、还原主备，原主机进行switchover成功
    9、执行完成后，关闭极致RTO成功
    10、查询主备数据一致
History     :
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified：2021-2-5 by 5328113;guc参数还原时指定还原值
    modified:2020-11-18 by @peilinqian ;修改kill-18后的断言；可能存在Rl状态；完善TPCC断言；
    modified:2020-12-17 by @peilinqian ;teardown增加主备异常恢复；异常恢复kill -18
    modified:2020-12-17 by @peilinqian ;删除kill-19、kill -18后的断言；
        变化的状态可能很多状态变化时间也不一定，只需保证后续恢复后功能正常即可；
    modified:2021-1-28 by 5328113;删除start成功断言，删除不必要tpcc
    modified:2024-7-10 by peilinqian;查询状态增加断言
"""

import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import *
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info('----Opengauss_Reliability_RTO_Dbsys_Case027:初始化----')

        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.tpcc_path = macro.TPCC_PATH
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            self.log.info('----1、开启极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主备是否同步----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----2、备机进行switchover----')
            switch_thread = ComThread(self.sta_1_sh.executeGsctl,
                                      args=('switchover',
                                            self.constant.SWITCH_SUCCESS_MSG,
                                            '',
                                            False))
            switch_thread.setDaemon(True)
            switch_thread.start()

            self.log.info('----3、主机数据库进程挂住kill - 19；----')
            self.com.kill_pid_keyword('M primary', 19, self.pri_dbuser)
            time.sleep(1)
            self.log.info('----查询主机数据数据进程状态----')
            db_process_cmd = 'ps ux | grep \'M primary\' | grep -v grep'
            self.log.info(db_process_cmd)
            db_process_result = self.pri_dbuser.sh(db_process_cmd).result()
            self.log.info(db_process_result)

            self.log.info('----查询主备状态----')
            time.sleep(5)
            status = self.pri_sh.getDbClusterStatus('detail')
            self.assertIn(self.constant.standby_need_repair, status)

            self.log.info('----4、switchover结果验证----')
            # 等待switchover完成
            switch_thread.join()
            switch_result = switch_thread.get_result()
            self.log.info(switch_result)
            self.assertFalse(switch_result)

            self.log.info('----5、等待30S，恢复主机数据库进程kill - 18；----')
            time.sleep(30)
            self.com.kill_pid_keyword('M primary', 18, self.pri_dbuser)
            time.sleep(1)
            self.log.info('----查询主机数据数据进程状态----')
            db_process_cmd = 'ps ux | grep \'M primary\' | grep -v grep'
            self.log.info(db_process_cmd)
            db_process_result = self.pri_dbuser.sh(db_process_cmd).result()
            self.log.info(db_process_result)

            self.log.info('----6、switchover失败后，重启主备；----')
            is_stop = self.pri_sh.stopDbCluster()
            self.assertTrue(is_stop)
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主备是否同步----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----7、备机再次进行switchover----')
            success_msg = self.constant.SWITCH_SUCCESS_MSG
            switch_result = self.sta_1_sh.executeGsctl('switchover',
                                                       success_msg)
            self.assertTrue(switch_result)

            self.log.info('----进行refresh----')
            refresh_msg = self.sta_1_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg)

            self.log.info('----查看主机query，同步是否正常----')
            self.sta_1_sh.check_location_consistency('primary',
                                                     self.node_num,
                                                     self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----8、还原主备，原主机进行switchover----')
            success_msg = self.constant.SWITCH_SUCCESS_MSG
            switch_result = self.pri_sh.executeGsctl('switchover',
                                                     success_msg)
            self.assertTrue(switch_result)

            self.log.info('----进行refresh----')
            refresh_msg = self.pri_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg)

            self.log.info('----查看主备是否同步----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

        finally:
            self.log.info('----9、执行完成后，关闭极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            # 校验备机是否完成数据同步
            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2)

            self.log.info('----10、查询主备数据是否一致----')
            nodes_tuple = (self.pri_dbuser,
                           self.sta1_dbuser,
                           self.sta2_dbuser)
            flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
            self.assertTrue(flag)

            table_dict = self.com.format_sql_result(
                self.pri_sh.executDbSql(r'\d'))
            table_name = table_dict.get('Name')
            for name in table_name:
                select_sql = f'select count(*) from {name};'
                flag = self.com.check_data_sample_by_all(select_sql,
                                                         *nodes_tuple)
                self.assertTrue(flag)

    def tearDown(self):
        self.log.info('----异常中断时，恢复数据库进程kill - 18----')
        self.com.kill_pid_keyword('M primary', 18, self.pri_dbuser)
        time.sleep(1)
        self.log.info('----查询主机数据数据进程状态----')
        db_process_cmd = 'ps ux | grep \'M primary\' | grep -v grep'
        self.log.info(db_process_cmd)
        db_process_result = self.pri_dbuser.sh(db_process_cmd).result()
        self.log.info(db_process_result)

        self.log.info('----恢复主备关系-----------------')
        host_name = self.pri_root.sh('hostname').result()
        result = self.pri_sh.check_whether_need_switch(host_name)
        if result:
            success_msg = self.constant.SWITCHOVER_SUCCESS_MSG
            result = self.pri_sh.executeGsctl('switchover', success_msg)
            self.log.info(result)
            result = self.pri_sh.exec_refresh_conf()
            self.log.info(result)

        self.log.info('----校验刷脏页是否完成----')
        self.pri_sh.check_whether_remain_dirty_page()

        self.log.info('----Opengauss_Reliability_RTO_Dbsys_Case027:执行完毕----')
